# Updated by Wei on 20140818 at school
print "Begins..."
# Pre process
numeratorTupleList = []
denominatorTupleList = []
minScore = 0.001
stepGap = 1.1
for i in range(0,110):
    print i,minScore
    minScore = minScore * 1.1
    currentTuple = (minScore,0)
    numeratorTupleList.append(currentTuple)
    denominatorTupleList.append(currentTuple)

print "Overall:"
print len(numeratorTupleList)
print len(denominatorTupleList)
# exit(1)

# Pass1:
# step1:
# analyze the file
NUM_QUERIES_NEEDED_TO_PROCESS = 100000
FIXED_THRESHOLD = 10
num_queries_processed = 0
qidANDTOP100ResultThresholdDict = {}
qid = 0
thresholdForResult = 9999

# option1:
# tail5K
# ifn = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/rawResults_tail5K_TOP100_withPartialImpactScores_20140722.txt"
# option2:
# head95K
ifn = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/raw_TOP1000_results_for_head_95K_queries_output_from_IRTK"
ifh = open(ifn,"r")
l = ifh.readline()
while l:
    le = l.strip().split(" ")
    if l.strip().startswith("qid:"):
        print qid
        print thresholdForResult
        print
        if qid not in qidANDTOP100ResultThresholdDict:
            qidANDTOP100ResultThresholdDict[qid] = thresholdForResult
        qid = le[1]
        thresholdForResult = 9999
        
        num_queries_processed += 1
        if num_queries_processed == NUM_QUERIES_NEEDED_TO_PROCESS:
            break
    if len(le) == 35 and le[-1].startswith("GX") and int(le[0]) <= FIXED_THRESHOLD:
        currTotalScore = float(le[-3])
        thresholdForResult = currTotalScore
    l = ifh.readline() 
ifh.close()

print qid
print thresholdForResult
print
if qid not in qidANDTOP100ResultThresholdDict:
    qidANDTOP100ResultThresholdDict[qid] = thresholdForResult

print "Overall:"
print "len(qidANDTOP100ResultThresholdDict):",len(qidANDTOP100ResultThresholdDict)
# print "qidANDTOP100ResultThresholdDict:",qidANDTOP100ResultThresholdDict
print "1:",qidANDTOP100ResultThresholdDict["1"]
# print "2:",qidANDTOP100ResultThresholdDict["2"]
# exit(1)

# Pass2:
totalPostingCounted = 0
num_queries_processed = 0
# option1:
# tail5K
# ifn = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/rawResults_tail5K_TOP100_withPartialImpactScores_20140722.txt"
# option2:
# head95K
ifn = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/raw_TOP1000_results_for_head_95K_queries_output_from_IRTK"
ifh = open(ifn,"r")
l = ifh.readline()
while l:
    le = l.strip().split(" ")
    if l.strip().startswith("qid:"):
        #print qid
        #print thresholdForResult
        #print
        qid = le[1]
        thresholdForResult = 9999
        num_queries_processed += 1
        if num_queries_processed == NUM_QUERIES_NEEDED_TO_PROCESS:
            break 
    if len(le) == 35 and le[-1].startswith("GX") and int(le[0]) <= 10:
        currTotalScore = float(le[-3])
        #print "qid:",qid
        #print "threshold:",qidANDTOP100ResultThresholdDict[qid]
        #print "currTotalScore:",currTotalScore
        #print
        positionBaseIndex = 1
        for i in range(0,10):
            currPostingScore = float(le[positionBaseIndex + i])
            bucketNum = -1
            # print currPostingScore
            if currPostingScore != 0.0:
                for j in range(0,len(numeratorTupleList)):
                    (currentBucketLowerBound,_) = numeratorTupleList[j]
                    (nextBucketLowerBound,_) = numeratorTupleList[j+1]
                    if currPostingScore >= currentBucketLowerBound and currPostingScore < nextBucketLowerBound:
                         # print j,currentBucketLowerBound,nextBucketLowerBound
                         break
                
                (numeratorLowerBound, numeratorCounter) = numeratorTupleList[j]
                (denominatorLowerBound, denominatorCounter) = denominatorTupleList[j]
                denominatorCounter += 1
                totalPostingCounted += 1
                
                # debug
                # print currTotalScore - currPostingScore
                # print qidANDTOP100ResultThresholdDict[qid]
                # print
                
                if (currTotalScore - currPostingScore) < qidANDTOP100ResultThresholdDict[qid]:
                    # essential
                    numeratorCounter += 1
                    # print "--->",currPostingScore,"essential"
                    pass
                else:
                    # not essential
                    # print "--->",currPostingScore,"NOT essential"
                    pass
                numeratorTupleList[j] = (numeratorLowerBound, numeratorCounter) 
                denominatorTupleList[j] = (denominatorLowerBound, denominatorCounter)
            else:
                pass
    l = ifh.readline()
ifh.close()

print "Overall:"
for i in range(0,len(numeratorTupleList)):
    (_,numeratorCounter) = numeratorTupleList[i]
    (_,denominatorCounter) = denominatorTupleList[i]
    if denominatorCounter == 0:
        print i,numeratorCounter,denominatorCounter,0.0
    else:
        print i,numeratorCounter,denominatorCounter,float(numeratorCounter)/float(denominatorCounter)
print "totalPostingCounted:",totalPostingCounted
print "Ends."
exit(1)

# step2:
# build the file
print "Begins..."
ifn = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/queryProcessingCostAnalysis/gov2/rawResults_50%_TOP1000_OR_20140126Night"
ifh = open(ifn,"r")
cl = ifh.readline()
numQ = 0
while cl:
    cle = cl.strip().split(" ")
    
    if cl.startswith("qid:"):
        print cl.strip()
        print ifh.readline().strip()
        print ifh.readline().strip()
        numQ += 1
        #if numQ == 2:
        #    break
    
    if len(cle) == 25:
        if int(cle[0]) <=100:
            print cle[0],
            baseIndex = 11
            for i in range(0,10):
                print cle[baseIndex + i],
            print cle[-4],cle[-3],cle[-2]
    
    if cl.startswith("Showing"):
        print cl.strip()
        print
    
    cl = ifh.readline()
ifh.close()
print "DONE."
print "numQ:",numQ
print "Ends."